package main

import (
	//"bufio"
	"bytes"
	"code.google.com/p/go.text/encoding/simplifiedchinese"
	"code.google.com/p/go.text/transform"
	"flag"
	"fmt"
	goq "github.com/PuerkitoBio/goquery"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	//"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	main_domain = "" //主域名
	waitgroup   = &sync.WaitGroup{}
	LOG         = ""
	ErrLOG      = ""
)

func main() {
	var citycode = flag.String("citycode", "qd", "请输入城市编号") //城市编号
	flag.Parse()
	main_domain = "http://" + *citycode + ".lianjia.com"
	LOG = *citycode + ".lianjia.com.out.log"
	ErrLOG = *citycode + ".lianjia.com.err.log"

	tmpurl := fmt.Sprintf("%s/xiaoqu/", main_domain)
	doc, e := getDoc(tmpurl)
	if e != nil {
		fmt.Println(e.Error())
		os.Exit(-1)
	}

	tmpRecordCount := filterStr(doc.Find(".list-head span").Text())
	RecordCount, e := strconv.Atoi(tmpRecordCount)
	if e != nil {
		os.Exit(-1)
	}
	PageSize := 30
	pageCount := RecordCount / PageSize
	if RecordCount%PageSize > 0 {
		pageCount++
	}

	for i := 1; i <= pageCount; i += 5 {
		for curpage := i; curpage < i+5 && curpage <= pageCount; curpage++ {
			waitgroup.Add(1)
			go getCommunityList(curpage)
		}
		waitgroup.Wait()
		time.Sleep(time.Second * 5)
	}
	fmt.Println("done.")
}

//获取楼盘列表
func getCommunityList(curpage int) {
	defer waitgroup.Done()
	tmpurl := fmt.Sprintf("%s/xiaoqu/pg%d/", main_domain, curpage)
	doc, e := getDoc(tmpurl)
	if e != nil {
		fmt.Println(e.Error())
		return
	}
	doc.Find("#house-lst").Children().Each(func(i int, sec *goq.Selection) {
		var (
			communityname    string //楼盘名称
			district, block  string //区域、板块
			salenum, rentnum string //出售、出租条数
			avgprice         string //均价
			link             string //链接
		)

		link, ok := sec.Find(".info-panel h2 a").Attr("href")
		if !ok {
			return
		}

		communityname = sec.Find(".info-panel h2").Text()
		if "" == communityname {
			communityname = "--"
		}
		rentnum = filterStr(strings.Replace(sec.Find(".col-1 .laisuzhou").Text(), "套正在出租", "", -1))
		if "" == rentnum {
			rentnum = "--"
		}
		tmpa := sec.Find(".col-1 .con a")
		if tmpa.Length() == 2 {
			district = filterStr(tmpa.Eq(0).Text())
			block = filterStr(tmpa.Eq(1).Text())
		}
		if "" == district {
			district = "--"
		}
		if "" == block {
			block = "--"
		}
		salenum = filterStr(sec.Find(".col-2 .num").Text())
		if "" == salenum {
			salenum = "--"
		}
		avgprice = filterStr(sec.Find(".col-3 .num").Text())
		if "" == avgprice {
			avgprice = "--"
		}

		agentlist := getAgentList(link)
		agentnum := len(agentlist)
		for k, _ := range agentlist {
			tmpstr := fmt.Sprintf("%s||%s||%s||%s||%s||%s||%d||%s", communityname, district, block, salenum, rentnum, avgprice, agentnum, k)
			outlog(tmpstr)
		}
	})
}

//获取在此楼盘发布房源的人数
func getAgentList(url string) map[string]bool {
	agentlist := make(map[string]bool)
	//出售经纪人
	tmpurlesf := fmt.Sprintf("%s%sesf/", main_domain, url)
	setAgentList(tmpurlesf, agentlist)

	//出租经纪人
	tmpurlzf := fmt.Sprintf("%s%szf/", main_domain, url)
	setAgentList(tmpurlzf, agentlist)

	return agentlist
}

//获取经纪人列表
func setAgentList(url string, agentlist map[string]bool) {
	doc, e := getDoc(url)
	if e != nil {
		return
	}
	PageSize := 30
	RecordCount, e := strconv.Atoi(doc.Find(".list-head h2 span").Text())
	if e != nil {
		RecordCount = 0
	}

	pageCount := RecordCount / PageSize
	if RecordCount%PageSize > 0 {
		pageCount++
	}

	for i := 1; i <= pageCount; i++ {
		pageurl := fmt.Sprintf("%spg%d/", url, i)
		doc, e := getDoc(pageurl)
		if e != nil {
			continue
		}
		doc.Find(".house-lst").Children().Each(func(i int, sec *goq.Selection) {
			link, ok := sec.Find(".info-panel h2 a").Attr("href")
			if ok {
				doc2, e := getDoc(link)
				if e == nil {
					agentinfo := filterStr(doc2.Find(".agent-info .p-01 a").Text())
					agentinfo += "||" + filterStr(doc2.Find(".contact-panel .ft-num").Text())
					if !agentlist[agentinfo] {
						agentlist[agentinfo] = true
					}
				}
			}
		})
	}
}

//获取doc
func getDoc(url string) (*goq.Document, error) {
	res, e := getResp(url)
	if e != nil {
		return nil, e
	}
	doc, e := goq.NewDocumentFromResponse(res)
	if e != nil {
		return nil, e
	}
	return doc, nil
}

//获取response
func getResp(url string) (*http.Response, error) {
	req, e := http.NewRequest("GET", url, nil)
	req.Header.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
	//req.Header.Add("Accept-Encoding", "gzip,deflate,sdch")
	req.Header.Add("Accept-Language", "zh-CN,zh;q=0.8")
	req.Header.Add("Cache-Control", "max-age=0")
	req.Header.Add("Proxy-Connection", "keep-alive")
	req.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36")
	c := &http.Client{
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				deadline := time.Now().Add(25 * time.Second)
				c, err := net.DialTimeout(netw, addr, time.Second*20)
				if err != nil {
					return nil, err
				}
				c.SetDeadline(deadline)
				return c, nil
			},
		},
	}

	var resp *http.Response
	//尝试三次
	for retry := 0; retry < 3; retry++ {
		resp, e = c.Do(req)
		if e != nil {
			time.Sleep(time.Second * 1)
		} else {
			break
		}
	}
	if e != nil {
		return nil, e
	}
	return resp, nil
}

//gbk->utf8
func G2U(s string) string {
	ret, e := gbktoutf8([]byte(s))
	if e != nil {
		return ""
	}
	return string(ret)
}

//utf8->gbk
func U2G(s string) string {
	ret, e := utf8togbk([]byte(s))
	if e != nil {
		return ""
	}
	return string(ret)
}

func gbktoutf8(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewDecoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}
func utf8togbk(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewEncoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}

func filterStr(str string) string {
	str = strings.Replace(str, "\r\n", "", -1)
	str = strings.Replace(str, "\r", "", -1)
	str = strings.Replace(str, "\n", "", -1)
	str = strings.Replace(str, "|", "", -1)
	str = strings.TrimSpace(str)
	return str
}

func s(a ...interface{}) {
	fmt.Println(a)
}

//outlog
func outlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_log.txt", pwd, LOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}

//errlog
func errlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_err.txt", pwd, ErrLOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}
